package edu.tsinghua.oxy.engineer.service.impl;

import edu.tsinghua.oxy.common.utils.HttpUtils;
import edu.tsinghua.oxy.engineer.entity.GovInfo;
import edu.tsinghua.oxy.engineer.service.IHtmlParse;
import edu.tsinghua.oxy.engineer.service.IProxyService;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;

/**
 * @author sean
 * @version 1.0
 * @date 2021/7/29
 */
@Slf4j
@Service("htmlParseGov")
public class HtmlParseGov implements IHtmlParse<GovInfo> {

    @Resource
    private IProxyService proxyService;

    @Resource
    private HttpUtils httpUtils;

    /**
     * 政府数据网列表页解析
     * @param listUrl 列表页面url
     *
     * @return
     */
    @Override
    public List<String> parseListHtml(String listUrl) {
        log.info("列表url：{}", listUrl);
        List<String> resultList = new ArrayList<>();

        //get获取结果集
        Document document = null;
        try {
            document = Jsoup.connect(listUrl)
                    .proxy(proxyService.getProxy())
                    .userAgent(httpUtils.getAgent()).timeout(60000).get();
        } catch (Exception e) {
            log.warn("获取网页内容： 失败 ", e);
            return null;
        }

        int index = listUrl.indexOf("/", 10);
        String baseUrl = listUrl.substring(0, index);

        Elements elements = document.getElementsByClass("dataset-heading");
        for (Element element : elements) {
            String href = element.getElementsByTag("a").get(0).attr("href");
            log.info("{}", href);
            resultList.add(baseUrl + href);
        }

        return resultList;
    }

    /**
     * 数据详情页解析
     * @param infoUrl 详情页面url
     *
     * @return
     */
    @Override
    public GovInfo parseInfoHtml(String infoUrl) {
        log.info("数据详情url：{}", infoUrl);
        //get获取结果集
        Document document = null;
        try {
            document = Jsoup.connect(infoUrl)
                    .proxy(proxyService.getProxy())
                    .userAgent(httpUtils.getAgent()).timeout(60000).get();
        } catch (Exception e) {
            log.warn("获取网页内容： 失败 ", e);
            return null;
        }

        GovInfo govInfo = new GovInfo();
        //解析标题等信息
        String title = document.getElementsByAttributeValue("itemprop", "name").get(0).text().trim();
        String description = document.getElementsByAttributeValue("itemprop", "description").get(0).text().trim();
        govInfo.setTitle(title);
        govInfo.setDescription(description);

        Elements topics = document.getElementsByClass("secondary").get(0)
                .getElementsByClass("topics");
        if(null != topics && topics.size() > 0){
            String topic = topics.get(0).getElementsByTag("a").text().trim();
            govInfo.setTopic(topic);
        }

        //解析日期
        Elements tds = document.getElementsByClass("table-condensed").get(0)
                .getElementsByTag("td");
        govInfo.setCreateDate(tds.get(0).text());
        govInfo.setUpdateDate(tds.get(1).text());

        //解析下载链接
        Elements resourecItems = document.getElementById("dataset-resources")
                .getElementsByClass("resource-item");
        for (Element item : resourecItems) {
            String url = item.getElementsByTag("a").last().attr("href").trim();
            if("distribution".equals(item.attr("itemprop"))){
                govInfo.getDownloadUrlList().add(url);
            } else {
                govInfo.getPageUrlList().add(url);
            }
        }

        return govInfo;
    }

    public static void main(String[] args) {
        HtmlParseGov htmlParseGov = new HtmlParseGov();
        String baseListUrl = "https://catalog.data.gov/dataset?page=";

        List<String> infoList = htmlParseGov.parseListHtml(baseListUrl + 1);
        log.info("{}", infoList);
//        for (String url : infoList) {
//            GovInfo govInfo = htmlParseGov.parseInfoHtml(url);
//            log.info("{}", govInfo);
//        }

//        String url = "https://catalog.data.gov/dataset/solar-electric-programs-reported-by-nyserda-beginning-2000";
//        GovInfo govInfo = htmlParseGov.parseInfoHtml(url);
//        log.info("{}", govInfo);
    }
}
